buildDataObject <- function(searchTableLoc='./quickAnalysis/prodAlgoTable.csv',dataObjectOutputLoc='./quickAnalysis/searchData.RData'){
  
  require(XML)
  require(stringr)
  
  prodAlgoTable <- read.csv(searchTableLoc,quote="",stringsAsFactors=FALSE)

  if (file.exists(dataObjectOutputLoc)){
    
    # look if it needs to be updated
    load(dataObjectOutputLoc)
    rowsToRun <- prodAlgoTable$algoCode[!(prodAlgoTable$algoCode %in% searchData$algoCode) | prodAlgoTable$overwriteDestFile]
    # delete files to overwrite from output
    searchData <- searchData[!searchData$algoCode %in% rowsToRun,]
    prodAlgoTable <- prodAlgoTable[rowsToRun,]
    
  } else  searchData <- NULL
  
  newSearchData <- NULL
  for(r in 1:nrow(prodAlgoTable)){
    
    thisProdRow <- prodAlgoTable[r,]
    searchSummary <- xmlParse(str_c('http://api.indeed.com/ads/apisearch?',
                            'publisher=2487201386265292&',
                            'q=',thisProdRow$algo,'&limit=1&v=2'))
    
    totalCount <- as.numeric(xmlToDataFrame(getNodeSet(searchSummary, "//totalresults"),
                                            stringsAsFactors=FALSE)$text)
    
    if (totalCount > 0) totalPages <- ceiling(totalCount/25) else totalPages <- 0
    
    cat(str_c('\n\nPROCESSING ', totalCount,' RESULTS FOR  ', thisProdRow$productCode,
              '\n\nQ=',thisProdRow$algo,'\n\n'))
    progressBar <- txtProgressBar(max=totalPages,style=3)
    
    thisAlgoData_raw <- NULL
    for (p in 1:totalPages){
      
      start_i <- (p-1)*25 
      
      page_p <- str_c('http://api.indeed.com/ads/apisearch?',
                      'publisher=2487201386265292&',
                      'q=',thisProdRow$algo,'&limit=25&start=',start_i,'&v=2')
      
      results <- xmlParse(page_p)
      data <- xmlToDataFrame(getNodeSet(results, "//result"),stringsAsFactors=FALSE)
   
      #thisAlgoData <- rbind(thisAlgoData,data)
      thisAlgoData_raw <- rbind(thisAlgoData_raw,data.frame(
                                           vendorCode=thisProdRow$vendorCode,
                                           productCode=thisProdRow$productCode,
                                           domainCode=thisProdRow$domainCode,
                                           algoCode=thisProdRow$algoCode,
                                           company=data$company,
                                           jobkey=data$jobkey,
                                           loc=data$formattedLocation,
                                           postSource=data$source,stringsAsFactors=FALSE))
      
      setTxtProgressBar(progressBar,p)
          
    }
  
    thisAlgoData_u <- unique(thisAlgoData_raw)  
    cat(str_c('\n\n ... ', nrow(thisAlgoData_u),' of ', nrow(thisAlgoData_raw), ' are unique.'))
    thisAlgoData <- thisAlgoData_u[!str_trim(thisAlgoData_u$company)=='',]
    cat(str_c('\n\n ... removing ', sum(str_trim(thisAlgoData_u$company)==''),' rows with no company name.'))
    
    thisAlgoData$rank <- (nrow(thisAlgoData):1)/nrow(thisAlgoData)
    newSearchData <- rbind(newSearchData,thisAlgoData)                     
  }
  
  searchData <- rbind(searchData,newSearchData)
  save(searchData,file=dataObjectOutputLoc) 

}

quickReport.overview <- function(dataInputObjectLoc='./quickAnalysis/searchData.RData',outputFile='./quickAnalysis/outputCharts/dataSummary.txt'){

  require(plyr)
  require(stringr)
  require(reshape2)
  
  load(dataInputObjectLoc)
  
  # ALL DATA SUMMARY
  file.remove(outputFile)
  
  locPerComp <- dcast(searchData[,c('company','loc')],company + loc ~ .,length,value.var='loc')[,3]
  resPerComp <- dcast(searchData[,c('company','productCode')],company ~ .,length,value.var='productCode')[,2]
  
  prodPerComp <- dcast(searchData[,c('company','productCode')],company ~ productCode,length,value.var='productCode')[,-1]
  prodPerComp <- table(apply(prodPerComp,1,function(thisRow)sum(thisRow>0)))
  names(prodPerComp) <- str_c(names(prodPerComp),' products')
  
  #capture.output({
    cat(str_c('\nALL DATA SUMMARY\n----------------------------------------------------------------\n',
              'total # of product searches:  \t ', length(unique(searchData$algoCode)),'\n\n',
              'total # of unique results:    \t ',nrow(searchData),'\n',
              'total # of unique companies:  \t ',length(unique(searchData$company)),'\n\n',
              'distribution of companies, by # of products used...\n\n'))
    print(prodPerComp)
    cat(str_c('\n# of results per company (distribution)... \n\n'))
    print(summary(resPerComp))
    cat(str_c('\n# of locations per company...\n\n'))
    print(summary(locPerComp))
    cat(str_c('----------------------------------------------------------------\n'))
  #},file=outputFile)
  
  # SEARCH DATA SUMMARY
  
  
  d_ply(searchData,'algoCode',function(algoData){
    
    prodAlgoTable <- read.csv('./quickAnalysis/prodAlgoTable.csv',)
    searchQuery <- prodAlgoTable$algo[prodAlgoTable$productCode %in% algoData$productCode]
    
    locPerComp <- dcast(algoData[,c('company','loc')],company + loc ~ .,length,value.var='loc')[,3]
    resPerComp <- dcast(algoData[,c('company','productCode')],company ~ .,length,value.var='productCode')[,2]

   capture.output({
      cat(str_c('\nSEARCH DATA SUMMARY FOR\n',
                'Q=',searchQuery,'\n----------------------------------------------------------------\n',
                'total # of unique results:    \t ',nrow(algoData),'\n',
                'total # of unique companies:  \t ',length(unique(algoData$company)),'\n'))
      cat(str_c('\n# of results per company (distribution)... \n\n'))
                 print(summary(resPerComp))
      cat(str_c('\n# of locations per company...\n\n'))
                 print(summary(locPerComp))
      cat(str_c('----------------------------------------------------------------\n'))
    },file=outputFile,append=TRUE)
  })       
}

quickReport.byProduct <- function(dataInputObjectLoc='./quickAnalysis/searchData.RData',outputChartDir='./quickAnalysis/outputCharts/'){
  
  require(plyr)
  require(googleVis)
  require(stringr)
  
  load(dataInputObjectLoc)
  
  prodAlgoTable <- read.csv('./quickAnalysis/prodAlgoTable.csv',)
    
  prodCompSum <- dlply(searchData,c('productCode'),function(thisProd){
    compLoc <- ddply(thisProd,'company',function(thisComp){
      return(data.frame(uniqueLocations=length(unique(thisComp$loc))))
    })
    compTable <- as.data.frame(table(thisProd$company))
    compTable_topN <- compTable[order(compTable$Freq,decreasing=TRUE),][1:50,]
    names(compTable_topN) <- c('company','productHits')  
    output <- merge(compTable_topN,compLoc,by='company')
    output <- output[order(output$productHits,decreasing=TRUE),]
  },.progress='text')
  
  
  allCharts <- list()
  for(pr in 1:length(prodCompSum)){
    thisProdCode <- names(prodCompSum)[pr]
    titleStr <- str_c('The search query \"',prodAlgoTable$algo[prodAlgoTable$productCode %in% thisProdCode],'\"
                      -----> returns ', sum(prodCompSum[[pr]]$productHits), ' unique results, across ',
                      nrow(prodCompSum[[pr]]), ' companies and ', sum(prodCompSum[[pr]]$uniqueLocations),
                      ' locations.')
    totRows <- nrow(prodCompSum[[pr]])
    allCharts[[pr]] <- gvisBarChart(prodCompSum[[pr]],options=list(
      title=titleStr,legend='top', width=1200, height=(totRows*25+100), vAxis.textStyle='{fontSize: 10}'))
  }
  output <- allCharts[[1]]
  for (tc in 2:length(allCharts)) output <- gvisMerge(output,allCharts[[tc]])
  capture.output(print(output),file=str_c(outputChartDir,'00_productCharts.html'))
  plot(output)

}

quickReport.byCompany <- function(dataInputObjectLoc='./quickAnalysis/searchData.RData',outputChartDir='./quickAnalysis/outputCharts/'){
  
  require(plyr)
  require(googleVis)
  require(stringr)
  
  load(dataInputObjectLoc)
  
  prodAlgoTable <- read.csv('./quickAnalysis/prodAlgoTable.csv',)
  
  prodCompSum <- ddply(searchData,c('company'),function(thisComp){
    prodData <- ddply(thisComp,'productCode',function(thisProd){
      return(data.frame(productHits=nrow(thisProd),avgRating=mean(thisProd$rank)))
    })
    output <- data.frame(prodData,totHits=sum(prodData$productHits))
  })
 
  prodCompSum <- within(prodCompSum,confidenceLevel <- 2*productHits*(avgRating-0.5))
  prodCompSum <- prodCompSum[order(prodCompSum$totHits,prodCompSum$company,prodCompSum$productHits,decreasing=TRUE),c(-4,-5)]
  
  allCharts <- list()
  compList <- unique(prodCompSum$company)
 
  for(co in 1:length(compList)){
    thisDf <- prodCompSum[prodCompSum$company==compList[co],-1]
    totRows <- nrow(thisDf)
    allCharts[[co]] <- gvisBarChart(thisDf,options=list(
      title=toupper(compList[co]),hAxis.minValue=0,legend='top', width=800, height=(totRows*30+100)))
  }
  
  # maxNum <- length(allCharts)
  maxNum <- 200
  for(iter in seq(1,length(allCharts),100)){
    output <- allCharts[[iter]] 
    for (tc in (iter+1):(iter+99)){
      if(tc > length(allCharts)) break
      output <- gvisMerge(output,allCharts[[tc]])
    }
    capture.output(print(output),file=str_c(outputChartDir,'/',iter,'_companyChart.html'))
  }
  
  
}


# converts text that would normally go in the indeed.com search bar to a 
# formatted URL query to be used with q=<functionOutput>
indeed.buildQuery<- function(searchBarText){
  str1 <- str_replace_all(searchBarText,'\\(','%28')
  str2 <- str_replace_all(str1,'\\)','%29')
  str3 <- str_replace_all(str2,' ','+')
  str4 <- str_replace_all(str3,'\"','%22')
  str5 <- str_replace_all(str4,':','%3A')
  return(str5)
}
